#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/5/16 15:50
# @Author  : Leixu
# @Site    : 
# @File    : tornado_http_client.py
# @Software: PyCharm Community Edition
import copy
from multiprocessing import Process, Queue, Event, cpu_count

import tornado
from tornado import gen
from tornado.concurrent import Future
from tornado.httpclient import *
from luobocrawler.utils.user_agent import UserAgent
from . import BaseClient


class TornadoHttpClient(BaseClient):
    def __init__(self, url, config, end_event, response_list=[]):
        self.response_list = response_list
        self.config = config
        self.url = url
        self.MAX_CONCURRENT_REQUESTS = 50
        self.end_event = end_event
        self.running_request_list = []
        self.client = AsyncHTTPClient(max_clients=self.MAX_CONCURRENT_REQUESTS * 2)

    @gen.coroutine
    def fetch(self, url):
        client = AsyncHTTPClient()
        request_config = copy.deepcopy(self.config["request_config"])
        if self.config["random_ua"]:
            request_config["headers"]["User-Agent"] = UserAgent.get_one_random_ua()
        request = HTTPRequest(url, **request_config)
        while len(self.running_request_list) > self.MAX_CONCURRENT_REQUESTS:
            self.clear_done_task()
            yield gen.sleep(0.1)
        self.running_request_list.append(client.fetch(request, self.normal_callback))

    @gen.coroutine
    def run(self):
        if isinstance(self.url, str):
            self.fetch(self.url)
        if isinstance(self.url, list):
            for i in self.url:
                self.fetch(i)
        IOLoop.current().spawn_callback(self.watch_dog, self.done)

    def done(self):
        # 通知处理已经结束了请求
        self.end_event.set()
        print("EveryThing done")

    @gen.coroutine
    def watch_dog(self, callback):
        while True:
            yield gen.sleep(0.010)
            self.clear_done_task()
            if len(self.running_request_list) > 0:
                continue
            else:
                break
        callback()

    def clear_done_task(self):
        self.running_request_list = list(filter(lambda x: x.running(), self.running_request_list))

    def normal_callback(self, response):
        self.response_list.append(response)


@gen.coroutine
def main(url, config, response_list=[]):
    end_event = Event()
    thc = TornadoHttpClient(url, config=config, end_event=end_event, response_list=response_list)
    thc.run()
    while True:
        if end_event.is_set():
            break
        else:
            # 协程睡眠
            yield gen.sleep(0.001)


def run_this_by_threading(url, config):
    print(config)
    response_list = []
    io_loop = tornado.ioloop.IOLoop.current()
    io_loop.run_sync(lambda: main(url, config, response_list=response_list))
    return response_list
